# Updated on 2013/07/27 by Wei 

print "Program begins..."
# The following file is nice for differentiating the concept between the length_of_the_inverted_index and my definition of term_freq_in_the_collection
# The file has 3 columns: the 1st column is the query term.
                        # the 2ed column is the length of the inverted index
                        # the 3rd column is the term freq in the collection
termFreqInCollectionDict = {}
inputFileName0 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/freqOfQueryTermsInCollection.txt"
inputFileHandler0 = open(inputFileName0,"r")
for line in inputFileHandler0.readlines():
    lineElements = line.strip().split(" ")
    # 3 columns
    # index:0 query term
    # index:1 length of the inverted list
    # index:2 term_freq_in_collection
    queryTerm = lineElements[0]
    # The training file has already has that
    
    termFreqInCollection = lineElements[2]
    
    # print queryTerm,termFreqInCollection
    
    if queryTerm not in termFreqInCollectionDict:
        termFreqInCollectionDict[queryTerm] = termFreqInCollection
    
print "len(termFreqInCollectionDict):",len(termFreqInCollectionDict)


# inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Training_Set_2013_07_15_tfqAdded.train"
inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719_tfqAdded.train"
inputFileHandler1 = open(inputFileName1,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded.train"
outputFileHandler = open(outputFileName,"w")

# switch for ON/OFF
# If the file has a headline, add this. If not, comment it
oldHeadLine = inputFileHandler1.readline()
newHeadLine = oldHeadLine.strip() + " " + "term_freq_in_collection"
outputFileHandler.write(newHeadLine + "\n")

for line in inputFileHandler1.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[4]
    if queryTerm not in termFreqInCollectionDict:
        print "error,queryTerm:",queryTerm
        exit(1)
    else:
        newLine = line.strip() + " " + termFreqInCollectionDict[queryTerm] + "\n"
        outputFileHandler.write(newLine)

inputFileHandler0.close()
inputFileHandler1.close()  
outputFileHandler.close()
print "Overall Processing Statistics:"
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "outputFileName:",outputFileName
print "Program ends."